{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "fa447fa7-6f85-4c6c-8bf0-d5f5ad9f9e97",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This notebook has been updated from my book to use the latest openai package version! At the time, 1.6.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "c12182ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "from datetime import datetime\n",
    "import hashlib\n",
    "import re\n",
    "from random import sample\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "f0289c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "pinecone_key = os.environ.get('PINECONE_API_KEY')\n",
    "client = OpenAI(\n",
    "    api_key=os.environ.get(\"OPENAI_API_KEY\")\n",
    ")\n",
    "\n",
    "INDEX_NAME = 'semantic-search-idx'\n",
    "NAMESPACE = 'default'\n",
    "ENGINE = 'text-embedding-3-large'  # has size 3072"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "b9befee1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pinecone import Pinecone, ServerlessSpec\n",
    "\n",
    "pc = Pinecone(\n",
    "    api_key=pinecone_key\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "309bf5f7-5ff1-40a6-944b-824f3f945ba2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['semantic-search-serverless', 'semantic-search', 'semantic-search-index']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc.list_indexes().names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "536c49b0-4beb-4f31-8f42-10e174dbff0c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "cf330fac-59da-40ce-90ac-99d25bc1d263",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating index semantic-search-idx\n"
     ]
    }
   ],
   "source": [
    "if INDEX_NAME not in pc.list_indexes().names():\n",
    "    print(f'Creating index {INDEX_NAME}')\n",
    "    pc.create_index(\n",
    "        name=INDEX_NAME,  # The name of the index\n",
    "        dimension=3072,  # The dimensionality of the vectors\n",
    "        metric='cosine',  # The similarity metric to use when searching the index\n",
    "        spec=ServerlessSpec(\n",
    "            cloud='aws',\n",
    "            region='us-west-2'\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "3cf993a3-6d51-49f4-8968-60f654d6202d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3072, 2)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# helper functions to get lists of embeddings from the OpenAI API\n",
    "def get_embeddings(texts, engine=ENGINE):\n",
    "    response = client.embeddings.create(\n",
    "        input=texts,\n",
    "        model=engine\n",
    "    )\n",
    "    \n",
    "    return [d.embedding for d in list(response.data)]\n",
    "\n",
    "def get_embedding(text, engine=ENGINE):\n",
    "    return get_embeddings([text], engine)[0]\n",
    "    \n",
    "len(get_embedding('hi')), len(get_embeddings(['hi', 'hello']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ea70672a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pinecone.data.index.Index at 0x2adc3f290>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Store the index as a variable\n",
    "index = pc.Index(name=INDEX_NAME)\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "e8088892",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dimension': 3072,\n",
       " 'index_fullness': 0.0,\n",
       " 'namespaces': {},\n",
       " 'total_vector_count': 0}"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.describe_index_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "7f2fdfe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c8c5b4f30923402163322b36c733a613'"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_hash(s):\n",
    "    # Return the MD5 hash of the input string as a hexadecimal string\n",
    "    return hashlib.md5(s.encode()).hexdigest()\n",
    "\n",
    "my_hash('I love to hash it hash it')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "ecd86f51",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_for_pinecone(texts, engine=ENGINE):\n",
    "    # Get the current UTC date and time\n",
    "    now = datetime.utcnow()\n",
    "    \n",
    "    # Generate vector embeddings for each string in the input list, using the specified engine\n",
    "    embeddings = get_embeddings(texts, engine=engine)\n",
    "    \n",
    "    # Create tuples of (hash, embedding, metadata) for each input string and its corresponding vector embedding\n",
    "    # The my_hash() function is used to generate a unique hash for each string, and the datetime.utcnow() function is used to generate the current UTC date and time\n",
    "    return [\n",
    "        (\n",
    "            my_hash(text),  # A unique ID for each string, generated using the my_hash() function\n",
    "            embedding,  # The vector embedding of the string\n",
    "            dict(text=text, date_uploaded=now)  # A dictionary of metadata, including the original text and the current UTC date and time\n",
    "        ) \n",
    "        for text, embedding in zip(texts, embeddings)  # Iterate over each input string and its corresponding vector embedding\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "3e1b73f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID:   49f68a5c8493ec2c0bf489821c21fc3b \n",
      "LEN:  3072 \n",
      "META: {'text': 'hi', 'date_uploaded': datetime.datetime(2024, 1, 26, 16, 31, 33, 538674)}\n"
     ]
    }
   ],
   "source": [
    "texts = ['hi']\n",
    "\n",
    "_id, embedding, metadata = prepare_for_pinecone(texts)[0]\n",
    "\n",
    "print('ID:  ',_id, '\\nLEN: ', len(embedding), '\\nMETA:', metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e1b8f60-8623-4384-a2b0-47be030ddaf7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2127de2b-561b-403b-ada0-119ce838e19d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "bf47aabd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def upload_texts_to_pinecone(texts, namespace=NAMESPACE, batch_size=None, show_progress_bar=False):\n",
    "    # Call the prepare_for_pinecone function to prepare the input texts for indexing\n",
    "    total_upserted = 0\n",
    "    if not batch_size:\n",
    "        batch_size = len(texts)\n",
    "\n",
    "    _range = range(0, len(texts), batch_size)\n",
    "    for i in tqdm(_range) if show_progress_bar else _range:\n",
    "        batch = texts[i: i + batch_size]\n",
    "        prepared_texts = prepare_for_pinecone(batch)\n",
    "\n",
    "        # Use the upsert() method of the index object to upload the prepared texts to Pinecone\n",
    "        total_upserted = index.upsert(\n",
    "            vectors=prepared_texts,\n",
    "            namespace=namespace\n",
    "        )['upserted_count']\n",
    "       \n",
    "\n",
    "    return total_upserted\n",
    "\n",
    "# Call the upload_texts_to_pinecone() function with the input texts\n",
    "upload_texts_to_pinecone(texts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "748a612a-15a7-4f3a-83e9-6fb1c8ca6993",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dimension': 3072,\n",
       " 'index_fullness': 0.0,\n",
       " 'namespaces': {},\n",
       " 'total_vector_count': 0}"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.describe_index_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "e8ed7e8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '49f68a5c8493ec2c0bf489821c21fc3b',\n",
       "  'metadata': {'date_uploaded': '2024-01-26T16:31:35.495082', 'text': 'hi'},\n",
       "  'score': 0.732532382,\n",
       "  'values': []}]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def query_from_pinecone(query, top_k=3):\n",
    "    # get embedding from THE SAME embedder as the documents\n",
    "    query_embedding = get_embedding(query, engine=ENGINE)\n",
    "\n",
    "    return index.query(\n",
    "      vector=query_embedding,\n",
    "      top_k=top_k,\n",
    "      namespace=NAMESPACE,\n",
    "      include_metadata=True   # gets the metadata (dates, text, etc)\n",
    "    ).get('matches')\n",
    "\n",
    "query_from_pinecone('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "84a0871f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import hashlib\n",
    "\n",
    "def delete_texts_from_pinecone(texts, namespace=NAMESPACE):\n",
    "    # Compute the hash (id) for each text\n",
    "    hashes = [hashlib.md5(text.encode()).hexdigest() for text in texts]\n",
    "    \n",
    "    # The ids parameter is used to specify the list of IDs (hashes) to delete\n",
    "    return index.delete(ids=hashes, namespace=namespace)\n",
    "\n",
    "# delete our text\n",
    "delete_texts_from_pinecone(texts)\n",
    "\n",
    "# test that the index is empty\n",
    "query_from_pinecone('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "72729603",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[36661, 1070]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing the tiktoken library\n",
    "import tiktoken\n",
    "\n",
    "# Initializing a tokenizer for the 'cl100k_base' model\n",
    "# This tokenizer is designed to work with the 'ada-002' embedding model\n",
    "tokenizer = tiktoken.encoding_for_model(\"gpt-4\")\n",
    "\n",
    "# Using the tokenizer to encode the text 'hey there'\n",
    "# The resulting output is a list of integers representing the encoded text\n",
    "# This is the input format required for embedding using the 'ada-002' model\n",
    "tokenizer.encode('hey there')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbc147d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "84f34d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to split the text into chunks of a maximum number of tokens. Inspired by OpenAI\n",
    "def overlapping_chunks(text, max_tokens = 500, overlapping_factor = 5):\n",
    "    '''\n",
    "    max_tokens: tokens we want per chunk\n",
    "    overlapping_factor: number of sentences to start each chunk with that overlaps with the previous chunk\n",
    "    '''\n",
    "\n",
    "    # Split the text using punctuation\n",
    "    sentences = re.split(r'[.?!]', text)\n",
    "\n",
    "    # Get the number of tokens for each sentence\n",
    "    n_tokens = [len(tokenizer.encode(\" \" + sentence)) for sentence in sentences]\n",
    "    \n",
    "    chunks, tokens_so_far, chunk = [], 0, []\n",
    "\n",
    "    # Loop through the sentences and tokens joined together in a tuple\n",
    "    for sentence, token in zip(sentences, n_tokens):\n",
    "\n",
    "        # If the number of tokens so far plus the number of tokens in the current sentence is greater \n",
    "        # than the max number of tokens, then add the chunk to the list of chunks and reset\n",
    "        # the chunk and tokens so far\n",
    "        if tokens_so_far + token > max_tokens:\n",
    "            chunks.append(\". \".join(chunk) + \".\")\n",
    "            if overlapping_factor > 0:\n",
    "                chunk = chunk[-overlapping_factor:]\n",
    "                tokens_so_far = sum([len(tokenizer.encode(c)) for c in chunk])\n",
    "            else:\n",
    "                chunk = []\n",
    "                tokens_so_far = 0\n",
    "\n",
    "        # If the number of tokens in the current sentence is greater than the max number of \n",
    "        # tokens, go to the next sentence\n",
    "        if token > max_tokens:\n",
    "            continue\n",
    "\n",
    "        # Otherwise, add the sentence to the chunk and add the number of tokens to the total\n",
    "        chunk.append(sentence)\n",
    "        tokens_so_far += token + 1\n",
    "    if chunk:\n",
    "        chunks.append(\". \".join(chunk) + \".\")\n",
    "\n",
    "    return chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "43e01a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "\n",
    "# A textbook about insects\n",
    "text = urlopen('https://www.gutenberg.org/cache/epub/10834/pg10834.txt').read().decode()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "dfdac6ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "non-overlapping chunking approach has 17 documents with average length 476.7 tokens\n"
     ]
    }
   ],
   "source": [
    "split = overlapping_chunks(text, overlapping_factor=0)\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in split]) / len(split)\n",
    "print(f'non-overlapping chunking approach has {len(split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "1730b34b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "overlapping chunking approach has 24 documents with average length 477.4 tokens\n"
     ]
    }
   ],
   "source": [
    "overlapping_split = overlapping_chunks(text)\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in overlapping_split]) / len(overlapping_split)\n",
    "print(f'overlapping chunking approach has {len(overlapping_split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1381cc8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a738d972-8557-4916-9731-089b50b3e5d1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "1f956759-9e4f-4945-ad21-43252ded5474",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "overlapping chunking approach has 127 documents with average length 121.1 tokens\n"
     ]
    }
   ],
   "source": [
    "overlapping_split = overlapping_chunks(text, max_tokens=128, overlapping_factor=2)\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in overlapping_split]) / len(overlapping_split)\n",
    "print(f'overlapping chunking approach has {len(overlapping_split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "3a3364dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upload_texts_to_pinecone(overlapping_split, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "8511c945",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dimension': 3072,\n",
       " 'index_fullness': 0.0,\n",
       " 'namespaces': {'default': {'vector_count': 127}},\n",
       " 'total_vector_count': 127}"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.describe_index_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67ddd3e8-a1fa-4d28-bb2b-6e109b88e9c8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "13cc8bdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5d253b260c308e67cb3ebcff1282c352\t0.60\t\n",
      "\n",
      "When e\n",
      "39eb0968545f5a7094621fb1a940dfe9\t0.56\t They abou\n",
      "ff09a697b4f85c5a1f5a3288d0f1298b\t0.49\t\n",
      "\n",
      "[Illus\n"
     ]
    }
   ],
   "source": [
    "query = 'How many horns does a flea have?'\n",
    "\n",
    "results_from_pinecone = query_from_pinecone(query, top_k=3)\n",
    "\n",
    "for result_from_pinecone in results_from_pinecone:\n",
    "    print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{result_from_pinecone['metadata']['text'][:10]}\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25b8e942",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "414fc2d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results_from_pinecone(query, top_k=3, verbose=True):\n",
    "\n",
    "    results_from_pinecone = query_from_pinecone(query, top_k=top_k)\n",
    "    if not results_from_pinecone:\n",
    "        return []\n",
    "\n",
    "    if verbose:\n",
    "        print(\"Query:\", query)\n",
    "    \n",
    "    \n",
    "    final_results = []\n",
    "\n",
    "    if verbose:\n",
    "        print('Document ID (Hash)\\t\\tRetrieval Score\\tText')\n",
    "    for result_from_pinecone in results_from_pinecone:\n",
    "        final_results.append(result_from_pinecone)\n",
    "        if verbose:\n",
    "            print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{result_from_pinecone['metadata']['text'][:50]}\")\n",
    "\n",
    "    return final_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "d89bcc77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: How many horns does a flea have?\n",
      "Document ID (Hash)\t\tRetrieval Score\tText\n",
      "5d253b260c308e67cb3ebcff1282c352\t0.60\t\n",
      "\n",
      "When examined by a microscope, the flea is a p\n",
      "39eb0968545f5a7094621fb1a940dfe9\t0.56\t They abound in warm countries, particularly in th\n",
      "ff09a697b4f85c5a1f5a3288d0f1298b\t0.49\t\n",
      "\n",
      "[Illustration]\n",
      "\n",
      "\n",
      "This very troublesome litt\n"
     ]
    }
   ],
   "source": [
    "final_results = get_results_from_pinecone(query, top_k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "c018b64d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delete_texts_from_pinecone(overlapping_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "33c716a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dimension': 3072,\n",
       " 'index_fullness': 0.0,\n",
       " 'namespaces': {},\n",
       " 'total_vector_count': 0}"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.describe_index_stats()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79e1caf7",
   "metadata": {},
   "source": [
    "# BoolQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "62e28784",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset None/None to /Users/sinanozdemir/.cache/huggingface/datasets/parquet/boolq-150dc58bba7bf36d/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53eab3e701114f30a7f20b22fb1732b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c67ff878e5546f79e93a684710f767e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/3.69M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ac9efc8df384c2d813bf4b3032e2596",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/1.26M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5df5ec0b1a8b404091b90cf369a33f5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32a801222b5b45f4b8bbfd9a2c6044cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/9427 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "251e31b5f3b54bdaa073a1a73243f4db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/3270 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset parquet downloaded and prepared to /Users/sinanozdemir/.cache/huggingface/datasets/parquet/boolq-150dc58bba7bf36d/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f6d9a0ffaa74ca988667071c0daea88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from evaluate import load\n",
    "\n",
    "dataset = load_dataset(\"boolq\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "e6773457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'does ethanol take more energy make that produces',\n",
       " 'answer': False,\n",
       " 'passage': \"All biomass goes through at least some of these steps: it needs to be grown, collected, dried, fermented, distilled, and burned. All of these steps require resources and an infrastructure. The total amount of energy input into the process compared to the energy released by burning the resulting ethanol fuel is known as the energy balance (or ``energy returned on energy invested''). Figures compiled in a 2007 report by National Geographic Magazine point to modest results for corn ethanol produced in the US: one unit of fossil-fuel energy is required to create 1.3 energy units from the resulting ethanol. The energy balance for sugarcane ethanol produced in Brazil is more favorable, with one unit of fossil-fuel energy required to create 8 from the ethanol. Energy balance estimates are not easily produced, thus numerous such reports have been generated that are contradictory. For instance, a separate survey reports that production of ethanol from sugarcane, which requires a tropical climate to grow productively, returns from 8 to 9 units of energy for each unit expended, as compared to corn, which only returns about 1.34 units of fuel energy for each unit of energy expended. A 2006 University of California Berkeley study, after analyzing six separate studies, concluded that producing ethanol from corn uses much less petroleum than producing gasoline.\"}"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['validation'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "cf88d118-a252-4a7a-98f3-74ed4b5a859c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 103/103 [05:38<00:00,  3.28s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upload_texts_to_pinecone(dataset['validation']['passage'], batch_size=32, show_progress_bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "9802a9ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dimension': 3072,\n",
       " 'index_fullness': 0.0,\n",
       " 'namespaces': {'default': {'vector_count': 2938}},\n",
       " 'total_vector_count': 2938}"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.describe_index_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "69cefb09-0ef1-4f27-b79a-947a1ff73dc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3270"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset['validation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "c9b0cf38-5ae9-491d-a0fc-e9828a30aed3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2938"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(dataset['validation']['passage']))  # only 2,938 unique passages!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "c20f66e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is love never dies a sequel to phantom of the opera\n",
      "Query: is love never dies a sequel to phantom of the opera\n",
      "Document ID (Hash)\t\tRetrieval Score\tText\n",
      "2c59ec305198d6689105532a6b3f11c3\t0.72\tLove Never Dies is a romantic musical composed by \n",
      "cb4349dac657f53542ffa36352ea344d\t0.48\tThe Phantom of the Opera is a musical with music b\n",
      "f52061f096d0d3c7e40c05f8f77e7305\t0.43\tLater, the freak show trio takes Gustave to meet t\n"
     ]
    }
   ],
   "source": [
    "query = sample(dataset['validation']['question'], 1)[0]\n",
    "print(query)\n",
    "final_results = get_results_from_pinecone(query, top_k=3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "e1e27222",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2c59ec305198d6689105532a6b3f11c3'"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_to_hash = {data['question']: my_hash(data['passage']) for data in dataset['validation']}\n",
    "\n",
    "q_to_hash[query]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61b9b10f-0e66-455c-b2f1-0bd7f06b6380",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "735bd234-556b-4cab-a2a4-a63ba4cf36d3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "a63ec53e",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_sample = dataset['validation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "bfea2ee7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:03<00:00,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of retrieval: 0.9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "predictions = []\n",
    "\n",
    "for question in tqdm(val_sample['question'][:10]):  # this would take too long..\n",
    "    retrieved_hash = get_results_from_pinecone(question, top_k=1, verbose=False)[0]['id']\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append(retrieved_hash == correct_hash)\n",
    "\n",
    "openai_accuracy = sum(predictions)/len(predictions)\n",
    "\n",
    "print(f'Accuracy of retrieval: {openai_accuracy}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "deb6353c-497d-4cec-bdd7-76bb42e90077",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 103/103 [00:58<00:00,  1.76it/s]\n"
     ]
    }
   ],
   "source": [
    "document_embeddings = []\n",
    "documents = val_sample['passage']\n",
    "\n",
    "for i in tqdm(list(range(0, len(documents), 32))):\n",
    "    batch = documents[i: i + 32]\n",
    "    document_embeddings += get_embeddings(batch)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "70bd2091-2e43-4ed4-9383-774db9cb59ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: when was halo 2 realsed?\n",
      "Document ID (Hash)\t\tRetrieval Score\tText\n",
      "10e8faab3d30566f4541b297a8d9e206\t0.54\tOn February 9, 2006, Nick Baron announced that a v\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'id': '10e8faab3d30566f4541b297a8d9e206',\n",
       "  'metadata': {'date_uploaded': '2024-01-26T16:38:29.348657',\n",
       "               'text': 'On February 9, 2006, Nick Baron announced that a '\n",
       "                       'version of Halo 2 would be released on PC, exclusively '\n",
       "                       'for the Windows Vista operating system. While this was '\n",
       "                       'a deliberate decision by Microsoft to push sales of '\n",
       "                       'Vista, the game could be enabled to play on Windows XP '\n",
       "                       'through an unauthorized third-party patch. The game was '\n",
       "                       'ported by a small team at Microsoft Game Studios '\n",
       "                       '(codenamed Hired Gun) who worked closely with Bungie. '\n",
       "                       'As one of the launch titles of Games for Windows -- '\n",
       "                       'Live, the game offered Live features not available in '\n",
       "                       'the Xbox version, such as guide support and '\n",
       "                       'achievements. The Windows port also added two exclusive '\n",
       "                       'multiplayer maps and a map editor.'},\n",
       "  'score': 0.540484667,\n",
       "  'values': []}]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_results_from_pinecone('when was halo 2 realsed?', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "677ed5a0-c893-40cf-a6e9-e69e18fd5485",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "ac35d596-478e-408d-9f10-cbeed8badb97",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "def simulate_top_pinecone_query(query):\n",
    "    query_embedding = get_embedding(query)\n",
    "    \n",
    "    # Concatenate the query vector with document vectors\n",
    "    all_embeddings = np.vstack([query_embedding] + document_embeddings)\n",
    "    \n",
    "    # Compute the cosine similarity matrix\n",
    "    similarity_matrix = cosine_similarity(all_embeddings)\n",
    "    \n",
    "    # Retrieve similarities for the query (first row, skipping the first element which is the query self-comparison)\n",
    "    query_similarities = similarity_matrix[0, 1:]\n",
    "    \n",
    "    # Find the index of the top document\n",
    "    top_document_score , top_document_index = np.max(query_similarities), np.argmax(query_similarities)\n",
    "    top_document = documents[top_document_index]\n",
    "    \n",
    "    return top_document_score, top_document_index, my_hash(top_document), top_document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "2b156fb4-973f-4ad0-baa0-2df92894620b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5404846319230268,\n",
       " 3034,\n",
       " '10e8faab3d30566f4541b297a8d9e206',\n",
       " 'On February 9, 2006, Nick Baron announced that a version of Halo 2 would be released on PC, exclusively for the Windows Vista operating system. While this was a deliberate decision by Microsoft to push sales of Vista, the game could be enabled to play on Windows XP through an unauthorized third-party patch. The game was ported by a small team at Microsoft Game Studios (codenamed Hired Gun) who worked closely with Bungie. As one of the launch titles of Games for Windows -- Live, the game offered Live features not available in the Xbox version, such as guide support and achievements. The Windows port also added two exclusive multiplayer maps and a map editor.')"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simulate_top_pinecone_query('when was halo 2 realsed?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "0cfebd73",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3270/3270 [43:12<00:00,  1.26it/s]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "for query in tqdm(val_sample['question']):\n",
    "    top_document_score, top_document_index, retrieved_hash, top_document = simulate_top_pinecone_query(query)\n",
    "    correct_hash = q_to_hash[query]\n",
    "    \n",
    "    openai_predictions.append(retrieved_hash == correct_hash)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "c1b3e254-2e68-491f-aaf8-9024094a285f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of retrieval: 0.8825688073394495\n"
     ]
    }
   ],
   "source": [
    "openai_accuracy = sum(openai_predictions)/len(openai_predictions)\n",
    "\n",
    "print(f'Accuracy of retrieval: {openai_accuracy}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f14809e",
   "metadata": {},
   "source": [
    "# OPEN SOURCE ALTERNATIVE TO EMBEDDING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "99138055",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4fbec6c1ef5463da9174605cedbd222",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(2, 768)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')\n",
    "\n",
    "docs = [\"Around 9 Million people live in London\", \"London is known for its financial district\"]\n",
    "\n",
    "doc_emb = model.encode(docs, batch_size=32, show_progress_bar=True)\n",
    "\n",
    "doc_emb.shape  #  == ('2, 768')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "be37c48b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f53de3dc97642ca907cce42aac1d014",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/103 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Encode query and documents\n",
    "docs = dataset['validation']['passage']\n",
    "doc_emb = model.encode(docs, batch_size=32, show_progress_bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ae0afee7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "can carbon 14 dating be used to detect the age of a live animal\n",
      "Query: can carbon 14 dating be used to detect the age of a live animal\n",
      "Document ID (Hash)\t\tRetrieval Score\tText\n",
      "04c16b27e72a49ff264c782230b3199f\t0.84\tThe method was developed in the late 1940s by Will\n",
      "e11909982c098d4b058616623548eadf\t0.73\tA fossil fuel is a fuel formed by natural processe\n",
      "7659a2874b2c633dee6b8227e95e9ac3\t0.72\tChimpanzees and other apes -- species which have b\n"
     ]
    }
   ],
   "source": [
    "query = sample(dataset['validation']['question'], 1)[0]\n",
    "print(query)\n",
    "final_results = get_results_from_pinecone(query, top_k=3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "45c71fb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5774650573730469 The method was developed in the late 1940s by Willard Libby, who received the Nobel Prize in Chemistry for his work in 1960. It is based on the fact that radiocarbon ( C) is constantly being created in the atmosphere by the interaction of cosmic rays with atmospheric nitrogen. The resulting C combines with atmospheric oxygen to form radioactive carbon dioxide, which is incorporated into plants by photosynthesis; animals then acquire C by eating the plants. When the animal or plant dies, it stops exchanging carbon with its environment, and from that point onwards the amount of C it contains begins to decrease as the C undergoes radioactive decay. Measuring the amount of C in a sample from a dead plant or animal such as a piece of wood or a fragment of bone provides information that can be used to calculate when the animal or plant died. The older a sample is, the less C there is to be detected, and because the half-life of C (the period of time after which half of a given sample will have decayed) is about 5,730 years, the oldest dates that can be reliably measured by this process date to around 50,000 years ago, although special preparation methods occasionally permit accurate analysis of older samples.\n",
      "0.37133681774139404 Carbon (from Latin: carbo ``coal'') is a chemical element with symbol C and atomic number 6. It is nonmetallic and tetravalent--making four electrons available to form covalent chemical bonds. It belongs to group 14 of the periodic table. Three isotopes occur naturally, C and C being stable, while C is a radionuclide, decaying with a half-life of about 5,730 years. Carbon is one of the few elements known since antiquity.\n",
      "0.29322588443756104 During the summer of 1990, a group of workers from the Black Hills Institute, located in Hill City, searched for fossils at the Cheyenne River Indian Reservation in western South Dakota near the city of Faith. By the end of the summer, the group had discovered Edmontosaurus bones and was ready to leave. However, a flat tire was discovered on their truck before the group could depart on August 12. While the rest of the group went into town to repair the truck, Sue Hendrickson decided to explore the nearby cliffs that the group had not checked. As she was walking along the base of a cliff, she discovered some small pieces of bone. She looked above her to see where the bones had originated, and observed larger bones protruding from the wall of the cliff. She returned to camp with two small pieces of the bones and reported the discovery to the president of the Black Hills Institute, Peter Larson. He determined that the bones were from a T. rex by their distinctive contour and texture. Later, closer examination of the site showed many visible bones above the ground and some articulated vertebrae. The crew ordered extra plaster and, although some of the crew had to depart, Hendrickson and a few other workers began to uncover the bones. The group was excited, as it was evident that much of the dinosaur had been preserved. Previously discovered T. rex skeletons were usually missing over half of their bones. It was later determined that Sue was a record 90 percent complete by bulk, and 73% complete counting the elements. Scientists believe that this specimen was covered by water and mud soon after its death which prevented other animals from carrying away the bones. Additionally, the rushing water mixed the skeleton together. When the fossil was found the hip bones were above the skull and the leg bones were intertwined with the ribs. The large size and the excellent condition of the bones were also surprising. The skull was 1,394 mm (54.9 in) long, and most of the teeth were still intact. After the group completed excavating the bones, each block was covered in burlap and coated in plaster, followed by a transfer to the offices of The Black Hills Institute where they began to clean the bones.\n"
     ]
    }
   ],
   "source": [
    "query_emb = model.encode(query)\n",
    "\n",
    "#Compute dot score between query and all document embeddings\n",
    "scores = util.dot_score(query_emb, doc_emb)[0].cpu().tolist()\n",
    "\n",
    "#Combine docs & scores\n",
    "doc_score_pairs = list(zip(docs, scores))\n",
    "\n",
    "#Sort by decreasing score\n",
    "doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True)\n",
    "\n",
    "#Output passages & scores\n",
    "for doc, score in doc_score_pairs[:3]:\n",
    "    print(score, doc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8533d9cb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "93280be8",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.setLevel(logging.CRITICAL)  # just to suppress some logs\n",
    "\n",
    "\n",
    "def eval_ranking_open_source(query, top_k=3):\n",
    "    query_emb = np.array(model.encode(query))\n",
    "\n",
    "    #Compute dot score between query and all document embeddings\n",
    "    scores = util.dot_score(query_emb, doc_emb)[0].cpu().tolist()\n",
    "\n",
    "    #Combine docs & scores\n",
    "    doc_score_pairs = list(zip(docs, scores))\n",
    "\n",
    "    #Sort by decreasing score\n",
    "    doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True)[:top_k]\n",
    "\n",
    "    retrieved_hash = my_hash(doc_score_pairs[0][0])\n",
    "    return retrieved_hash\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "06f198cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'04c16b27e72a49ff264c782230b3199f'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_ranking_open_source(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "8faf0c1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3270/3270 [02:46<00:00, 19.70it/s]\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "predictions = []\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash = eval_ranking_open_source(question, top_k=3)\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append(retrieved_hash == correct_hash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1eca7d89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Open-source accuracy: 0.8235474006116208\n"
     ]
    }
   ],
   "source": [
    "open_source_accuracy = sum(predictions)/len(predictions)\n",
    "\n",
    "print(f'Open-source accuracy: {open_source_accuracy}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1333c768-8af1-4533-9479-e6b8cb0821dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "083b2201-6327-45a9-8582-559cf3aca539",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4510f2d9-23c5-4fcf-b253-6a08b20bdce2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "384d5cc6-9465-4cc0-84cb-3ae5fc32ac84",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0ed10f2-0fa9-4bbd-8b43-73cad37572a6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3a12313-8514-4fea-9c19-4a55293948d5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "803615d0-ca38-4742-a0c8-9410aac78db5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3a986c9-c33b-4b40-b8fe-4a28819dbce4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e93fdab7-ca4b-47ec-ad4c-4116163c5536",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
